{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Vanilla CNN DQN"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "%load_ext autoreload\n",
    "%autoreload 2\n",
    "\n",
    "import matplotlib.pyplot as plt\n",
    "import gym\n",
    "\n",
    "import numpy as np\n",
    "import random\n",
    "from collections import namedtuple, deque\n",
    "from itertools import count\n",
    "import time\n",
    "import datetime\n",
    "\n",
    "import torch\n",
    "import torch.nn as nn\n",
    "import torch.nn.functional as F\n",
    "import torch.optim as optim\n",
    "\n",
    "import pdb"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "WARNING:root:This caffe2 python run does not have GPU support. Will run in CPU only mode.\n"
     ]
    }
   ],
   "source": [
    "from src.utils.Config import Config\n",
    "from src.utils.Logging import Logger\n",
    "from src.utils.atari_wrappers import wrap_deepmind, make_atari"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/Library/Frameworks/Python.framework/Versions/3.6/lib/python3.6/site-packages/gym/envs/registration.py:14: PkgResourcesDeprecationWarning: Parameters to load are deprecated.  Call .resolve and .require separately.\n",
      "  result = entry_point.load(False)\n"
     ]
    }
   ],
   "source": [
    "seed = 1234\n",
    "\n",
    "env_id = \"PongNoFrameskip-v4\"\n",
    "env    = make_atari(env_id)\n",
    "env    = wrap_deepmind(env, episode_life=True, clip_rewards=True, frame_stack=True, scale=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "observation shape: (84, 84, 4)\n",
      "Action space: 6\n"
     ]
    }
   ],
   "source": [
    "print(\"observation shape: {}\".format(env.observation_space.shape))\n",
    "print(\"Action space: {}\".format(env.action_space.n))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Parsing state from lazy frame to torch tensor"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "tensor([[[[0.2031, 0.2031, 0.2031,  ..., 0.3398, 0.3398, 0.3398],\n",
       "          [0.3398, 0.3398, 0.3398,  ..., 0.3398, 0.3398, 0.3398],\n",
       "          [0.3398, 0.3398, 0.3398,  ..., 0.3398, 0.3398, 0.3398],\n",
       "          ...,\n",
       "          [0.9219, 0.9219, 0.9219,  ..., 0.9219, 0.9219, 0.9219],\n",
       "          [0.9219, 0.9219, 0.9219,  ..., 0.9219, 0.9219, 0.9219],\n",
       "          [0.9219, 0.9219, 0.9219,  ..., 0.9219, 0.9219, 0.9219]],\n",
       "\n",
       "         [[0.2031, 0.2031, 0.2031,  ..., 0.3398, 0.3398, 0.3398],\n",
       "          [0.3398, 0.3398, 0.3398,  ..., 0.3398, 0.3398, 0.3398],\n",
       "          [0.3398, 0.3398, 0.3398,  ..., 0.3398, 0.3398, 0.3398],\n",
       "          ...,\n",
       "          [0.9219, 0.9219, 0.9219,  ..., 0.9219, 0.9219, 0.9219],\n",
       "          [0.9219, 0.9219, 0.9219,  ..., 0.9219, 0.9219, 0.9219],\n",
       "          [0.9219, 0.9219, 0.9219,  ..., 0.9219, 0.9219, 0.9219]],\n",
       "\n",
       "         [[0.2031, 0.2031, 0.2031,  ..., 0.3398, 0.3398, 0.3398],\n",
       "          [0.3398, 0.3398, 0.3398,  ..., 0.3398, 0.3398, 0.3398],\n",
       "          [0.3398, 0.3398, 0.3398,  ..., 0.3398, 0.3398, 0.3398],\n",
       "          ...,\n",
       "          [0.9219, 0.9219, 0.9219,  ..., 0.9219, 0.9219, 0.9219],\n",
       "          [0.9219, 0.9219, 0.9219,  ..., 0.9219, 0.9219, 0.9219],\n",
       "          [0.9219, 0.9219, 0.9219,  ..., 0.9219, 0.9219, 0.9219]],\n",
       "\n",
       "         [[0.2031, 0.2031, 0.2031,  ..., 0.3398, 0.3398, 0.3398],\n",
       "          [0.3398, 0.3398, 0.3398,  ..., 0.3398, 0.3398, 0.3398],\n",
       "          [0.3398, 0.3398, 0.3398,  ..., 0.3398, 0.3398, 0.3398],\n",
       "          ...,\n",
       "          [0.9219, 0.9219, 0.9219,  ..., 0.9219, 0.9219, 0.9219],\n",
       "          [0.9219, 0.9219, 0.9219,  ..., 0.9219, 0.9219, 0.9219],\n",
       "          [0.9219, 0.9219, 0.9219,  ..., 0.9219, 0.9219, 0.9219]]]])"
      ]
     },
     "execution_count": 5,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "def parse_state(state):\n",
    "    state = np.array(state).transpose((2, 0, 1))\n",
    "    state = torch.from_numpy(state)\n",
    "    state = state.unsqueeze(0).float() / 256\n",
    "    return state\n",
    "\n",
    "state = env.reset()\n",
    "parse_state(state)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Testing out Network"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "class CNNQNetwork(nn.Module):\n",
    "    def __init__(self, channel_in, action_size, seed, fc1_units=512, fc2_units=512):\n",
    "        # Call inheritance\n",
    "        super(CNNQNetwork, self).__init__()\n",
    "        self.seed = torch.manual_seed(1234)\n",
    "        \n",
    "        self.conv_head = nn.Sequential(\n",
    "            nn.Conv2d(4, 32, kernel_size=8, stride=4),\n",
    "            nn.BatchNorm2d(32),\n",
    "            nn.ReLU(),\n",
    "            nn.Conv2d(32, 64, kernel_size=4, stride=2),\n",
    "            nn.BatchNorm2d(64),\n",
    "            nn.ReLU(),\n",
    "            nn.Conv2d(64, 64, kernel_size=3, stride=1),\n",
    "            nn.BatchNorm2d(64),\n",
    "            nn.ReLU(),\n",
    "        )\n",
    "        \n",
    "        self.fc = nn.Sequential(\n",
    "            nn.Linear(64 * 7 * 7, fc1_units),\n",
    "            nn.ReLU(),\n",
    "            nn.Linear(fc1_units, action_size),\n",
    "        )\n",
    "\n",
    "    def forward(self, state):\n",
    "        x = self.conv_head(state)\n",
    "        x = x.view(x.size(0), -1) # Flatten\n",
    "        x = self.fc(x)\n",
    "        return x"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "torch.Size([1, 64, 7, 7])"
      ]
     },
     "execution_count": 7,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "state = parse_state(env.reset())\n",
    "\n",
    "conv_head = nn.Sequential(\n",
    "    nn.Conv2d(4, 32, kernel_size=8, stride=4),\n",
    "    nn.BatchNorm2d(32),\n",
    "    nn.ReLU(),\n",
    "    nn.Conv2d(32, 64, kernel_size=4, stride=2),\n",
    "    nn.BatchNorm2d(64),\n",
    "    nn.ReLU(),\n",
    "    nn.Conv2d(64, 64, kernel_size=3, stride=1),\n",
    "    nn.BatchNorm2d(64),\n",
    "    nn.ReLU(),\n",
    ")\n",
    "\n",
    "conv_head(state).shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "tensor([[ 0.0606,  0.1745,  0.0196,  0.1161, -0.2145,  0.1829]],\n",
       "       grad_fn=<AddmmBackward>)"
      ]
     },
     "execution_count": 8,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "model = CNNQNetwork(4, env.action_space.n, seed, 512)\n",
    "model(state)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## DQN Agent & Replay Buffer"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [],
   "source": [
    "class ReplayBuffer:\n",
    "    \"\"\"Fixed-size buffer to store experience tuples.\"\"\"\n",
    "\n",
    "    def __init__(self, config):\n",
    "        \"\"\"Initialize a ReplayBuffer object.\n",
    "\n",
    "        Params\n",
    "        ======\n",
    "        action_size (int): dimension of each action\n",
    "        buffer_size (int): maximum size of buffer\n",
    "        batch_size (int): size of each training batch\n",
    "        seed (int): random seed\n",
    "        \"\"\"\n",
    "\n",
    "        self.memory = deque(maxlen=config.buffer_size)  \n",
    "        self.batch_size = config.batch_size\n",
    "        self.experience = namedtuple(\"Experience\", field_names=[\"state\", \"action\", \"reward\", \"next_state\", \"done\"])\n",
    "        self.seed = random.seed(config.seed)\n",
    "        self.device = config.device\n",
    "\n",
    "    def add(self, state, action, reward, next_state, done, error=0):\n",
    "        \"\"\"Add a new experience to memory.\"\"\"\n",
    "        e = self.experience(state, action, reward, next_state, done)\n",
    "        self.memory.append(e)\n",
    "\n",
    "    def sample(self):\n",
    "        \"\"\"Randomly sample a batch of experiences from memory.\"\"\"\n",
    "        experiences = random.sample(self.memory, k=self.batch_size)\n",
    "        \n",
    "#         pdb.set_trace()\n",
    "\n",
    "        states = torch.from_numpy(np.vstack([e.state for e in experiences if e is not None])).float().to(self.device)\n",
    "        actions = torch.from_numpy(np.vstack([e.action for e in experiences if e is not None])).long().to(self.device)\n",
    "        rewards = torch.from_numpy(np.vstack([e.reward for e in experiences if e is not None])).float().to(self.device)\n",
    "        next_states = torch.from_numpy(np.vstack([e.next_state for e in experiences if e is not None])).float().to(self.device)\n",
    "        dones = torch.from_numpy(np.vstack([e.done for e in experiences if e is not None]).astype(np.uint8)).float().to(self.device)\n",
    "        weights = torch.ones(len(experiences)).float().to(self.device)\n",
    "        memory_loc = torch.zeros(len(experiences)).float().to(self.device)\n",
    "\n",
    "        return (states, actions, rewards, next_states, dones, weights, memory_loc)\n",
    "\n",
    "    def n_entries(self):\n",
    "        return len(self.memory)\n",
    "\n",
    "    def __len__(self):\n",
    "        \"\"\"Return the current size of internal memory.\"\"\"\n",
    "        return len(self.memory)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [],
   "source": [
    "class Agent():\n",
    "    def __init__(self, state_size, action_size, config):\n",
    "        \"\"\"\n",
    "        Initialize an Agent object.\n",
    "\n",
    "        state_size (int): dimension of each state\n",
    "        action_size (int): dimension of each action\n",
    "        \"\"\"\n",
    "\n",
    "        if config.model is None:\n",
    "            raise Exception(\"Please select a Model for agent\")\n",
    "        if config.memory is None:\n",
    "            raise Exception(\"Please select Memory for agent\") \n",
    "\n",
    "        self.config = config\n",
    "\n",
    "        self.state_size = state_size\n",
    "        self.action_size = action_size\n",
    "        self.seed = random.seed(self.config.seed)\n",
    "\n",
    "        # Q-Network\n",
    "        self.qnetwork_local = self.config.model(state_size, action_size, self.seed, 512, 512).to(self.config.device)\n",
    "        self.qnetwork_target = self.config.model(state_size, action_size, self.seed, 512, 512).to(self.config.device)\n",
    "\n",
    "        self.optimizer = optim.Adam(self.qnetwork_local.parameters(), lr=self.config.lr)\n",
    "\n",
    "        # Replay memory\n",
    "        self.memory = self.config.memory(config)\n",
    "\n",
    "        # Initialize time step (for updating every UPDATE_EVERY steps)\n",
    "        self.t_step = 0\n",
    "        \n",
    "        # Keep \n",
    "        self.eps = self.config.eps_start\n",
    "    \n",
    "    def step(self, state, action, reward, next_state, done):\n",
    "        state = self.parse_state(state)\n",
    "        next_state = self.parse_state(next_state)\n",
    "        \n",
    "        # Save experience in replay memory\n",
    "        self.memory.add(state, action, reward, next_state, done) \n",
    "\n",
    "        # Learn every UPDATE_EVERY time steps.\n",
    "        self.t_step = (self.t_step + 1) % self.config.learn_every\n",
    "\n",
    "        if self.t_step == 0:\n",
    "            # If enough samples are available in memory, get random subset and learn\n",
    "            if self.memory.n_entries() > self.config.batch_size:\n",
    "                experiences = self.memory.sample()\n",
    "                self.learn(experiences, self.config.gamma)\n",
    "\n",
    "    def anneal_eps(self):\n",
    "        self.eps = max(self.config.eps_end, self.config.eps_decay*self.eps) \n",
    "    \n",
    "    def parse_state(self, state):\n",
    "        state = np.array(state).transpose((2, 0, 1))\n",
    "        state = torch.from_numpy(state)\n",
    "        state = state.unsqueeze(0).float() / 256\n",
    "        return state\n",
    "    \n",
    "    def act(self, state, network_only=False):\n",
    "        state = self.parse_state(state).to(self.config.device)\n",
    "\n",
    "        self.qnetwork_local.eval()\n",
    "        with torch.no_grad():\n",
    "            action_values = self.qnetwork_local(state)\n",
    "        self.qnetwork_local.train()\n",
    "        \n",
    "        if network_only:\n",
    "            return np.argmax(action_values.cpu().data.numpy())\n",
    "        \n",
    "        # Epsilon-greedy action selection\n",
    "        if random.random() > self.eps:\n",
    "            return np.argmax(action_values.cpu().data.numpy())\n",
    "        else:\n",
    "            return random.choice(np.arange(self.action_size))\n",
    "    \n",
    "    def learn(self, experiences, gamma):\n",
    "        \"\"\"Update value parameters using given batch of experience tuples.\n",
    "\n",
    "        Params\n",
    "        ======\n",
    "        experiences (Tuple[torch.Tensor]): tuple of (s, a, r, s', done) tuples \n",
    "        gamma (float): discount factor\n",
    "        \"\"\"\n",
    "        states, actions, rewards, next_states, dones, is_weights, idxs = experiences\n",
    "\n",
    "        # Get expected Q values from local model\n",
    "        Q_expected = self.qnetwork_local(states).gather(1, actions)\n",
    "\n",
    "\n",
    "        # Get max action from network\n",
    "        max_next_actions = self.get_max_next_actions(next_states)\n",
    "\n",
    "\n",
    "        # Get max_next_q_values -> .gather(\"dim\", \"index\")\n",
    "        max_next_q_values = self.qnetwork_target(next_states).gather(1, max_next_actions)\n",
    "\n",
    "        # Y^q\n",
    "        Q_targets = rewards + (gamma * max_next_q_values * (1 - dones))\n",
    "\n",
    "        loss = F.mse_loss(Q_expected, Q_targets)\n",
    "\n",
    "        # Minimize the loss\n",
    "        # - Optimizer is initilaised with qnetwork_local, so will update that one.\n",
    "        self.optimizer.zero_grad()\n",
    "        loss.backward()\n",
    "        self.optimizer.step()\n",
    "\n",
    "        # ------------------- update target network ------------------- #\n",
    "        self.soft_update(self.qnetwork_local, self.qnetwork_target, self.config.tau)\n",
    "    \n",
    "    def soft_update(self, local_model, target_model, tau):\n",
    "        \"\"\"Soft update model parameters.\n",
    "        θ_target = τ*θ_local + (1 - τ)*θ_target\n",
    "\n",
    "        Params\n",
    "        ======\n",
    "        local_model (PyTorch model): weights will be copied from\n",
    "        target_model (PyTorch model): weights will be copied to\n",
    "        tau (float): interpolation parameter \n",
    "        \"\"\"\n",
    "        for target_param, local_param in zip(target_model.parameters(), local_model.parameters()):\n",
    "            target_param.data.copy_(tau*local_param.data + (1.0-tau)*target_param.data)\n",
    "    \n",
    "    def get_max_next_actions(self, next_states):\n",
    "        '''\n",
    "        Passing next_states through network will give array of all action values (in batches)\n",
    "        eg: [[0.25, -0.35], [-0.74, -0.65], ...]\n",
    "        - Detach will avoid gradients being calculated on the variables\n",
    "        - max() gives max value in whole array (0.25)\n",
    "        - max(0) gives max in dim=0 ([0.25, -0.35])\n",
    "        - max(1) gives max in dim=1, therefore: two tensors, one of max values and one of index\n",
    "        eg: \n",
    "        - values=tensor([0.25, -0.65, ...])\n",
    "        - indices=tensor([0, 1, ...])\n",
    "        - we'll want to take the [1] index as that is only the action\n",
    "        eg: tensor([0, 1, ...]) \n",
    "        - unsqueeze allows us to create them into an array that is usuable for next bit (.view(-1,1) would also work)\n",
    "        eg: eg: tensor([[0], [1], ...]) \n",
    "        '''\n",
    "        return self.qnetwork_target(next_states).detach().max(1)[1].unsqueeze(1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Agent Configuration:\n",
      "env: \t\tEnvSpec(PongNoFrameskip-v4)\n",
      "win condition: \t10000\n",
      "device: \tcpu\n",
      "seed: \t\t123456789\n",
      "n_episodes: \t2000\n",
      "max_t: \t\t1000\n",
      "eps_start: \t1.0\n",
      "eps_end: \t0.01\n",
      "eps_decay: \t0.995\n",
      "eps_greedy: \tTrue\n",
      "noisy: \t\tFalse\n",
      "tau: \t\t0.001\n",
      "gamma: \t\t0.99\n",
      "lr: \t\t0.0005\n",
      "memory: \t<class '__main__.ReplayBuffer'>\n",
      "batch_size: \t32\n",
      "buffer_size: \t100000\n",
      "lr_annealing: \tFalse\n",
      "learn_every: \t4\n",
      "double_dqn: \tFalse\n",
      "model: \t\t<class '__main__.CNNQNetwork'>\n",
      "save_loc: \tNone\n",
      "<_sre.SRE_Match object; span=(0, 27), match='EnvSpec(PongNoFrameskip-v4)'>\n",
      "Logging at: logs/PongNoFrameskip-v4/experiment-2020-05-25_16_49_11\n"
     ]
    }
   ],
   "source": [
    "config = Config()\n",
    "\n",
    "config.env = env\n",
    "\n",
    "config.win_condition = 10000\n",
    "n_episodes = 400\n",
    "config.memory = ReplayBuffer\n",
    "config.batch_size = 32\n",
    "config.model = CNNQNetwork\n",
    "config.print_config()\n",
    "\n",
    "logger = Logger(config)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [],
   "source": [
    "def train(config, logger):\n",
    "    experiment_start = time.time()\n",
    "    env = config.env\n",
    "    frame = 0\n",
    "\n",
    "    agent = Agent(4, action_size=env.action_space.n, config=config)\n",
    "\n",
    "    total_scores_deque = deque(maxlen=100)\n",
    "    total_scores = []\n",
    "\n",
    "\n",
    "    for i_episode in range(1, config.n_episodes+1):\n",
    "        states = env.reset()\n",
    "        scores = 0\n",
    "\n",
    "        start_time = time.time()\n",
    "\n",
    "        for t in count():\n",
    "            actions = agent.act(states)\n",
    "            next_states, rewards, dones, _ = env.step(np.array(actions))\n",
    "            agent.step(states, actions, rewards, next_states,dones)\n",
    "\n",
    "\n",
    "\n",
    "            states = next_states\n",
    "            scores += rewards\n",
    "            number_of_time_steps = t\n",
    "            frame += 1\n",
    "\n",
    "            if np.any(dones):\n",
    "                break \n",
    "\n",
    "        agent.anneal_eps()\n",
    "\n",
    "\n",
    "        # Book Keeping\n",
    "        mean_score = np.mean(scores)\n",
    "        min_score = np.min(scores)\n",
    "        max_score = np.max(scores)\n",
    "\n",
    "        total_scores_deque.append(mean_score)\n",
    "        total_scores.append(mean_score)\n",
    "        total_average_score = np.mean(total_scores_deque)\n",
    "\n",
    "        logger.log_scalar(\"score\", mean_score, i_episode)\n",
    "        logger.log_scalar(\"average_score\", total_average_score, i_episode)\n",
    "\n",
    "        duration = time.time() - start_time\n",
    "\n",
    "        # print('\\rEpisode {}\\tTotal Average Score (in 100 window): {:.4f}\\tMean: {:.4f}\\tMin: {:.4f}\\tMax: {:.4f}\\tDuration: {:.4f}\\t#TimeSteps: {:.4f}'.format(i_episode, total_average_score, mean_score, min_score, max_score, duration, number_of_time_steps), end=\"\")\n",
    "        print('\\rEpi: {}\\t Frame: {} \\tAverage: {:.4f}\\tMean: {:.4f}\\tDuration: {:.2f}\\t#t_s: {:.1f}'.format(i_episode, frame, total_average_score, mean_score, duration, number_of_time_steps), end=\"\")\n",
    "\n",
    "        if i_episode % 100 == 0:\n",
    "            print('\\rEpi: {}\\t Frame: {}\\tAverage Score: {:.4f}\\tMean: {:.4f}\\tDuration: {:.2f}\\t#t_s: {:.1f}'.format(i_episode, frame, total_average_score, mean_score, duration, number_of_time_steps))\n",
    "            torch.save(agent.qnetwork_local.state_dict(), \"{}/checkpoint.pth\".format(logger.log_file_path, date=datetime.datetime.now()))\n",
    "        if config.win_condition is not None and total_average_score > config.win_condition: \n",
    "            print(\"\\nEnvironment Solved in {:.4f} seconds !\".format(time.time() - experiment_start))\n",
    "            torch.save(agent.qnetwork_local.state_dict(), \"{}/checkpoint.pth\".format(logger.log_file_path, date=datetime.datetime.now()))\n",
    "            return \n",
    "\n",
    "    return"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 105,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "checkpoint-score17_56.pth\n",
      "Epi: 100\t Frame: 95038\tAverage Score: -20.1400\tMean: -18.0000\tDuration: 9.08\t#t_s: 1064.0\n",
      "Epi: 200\t Frame: 215265\tAverage Score: -18.6400\tMean: -18.0000\tDuration: 10.16\t#t_s: 1219.0\n",
      "Epi: 300\t Frame: 362446\tAverage Score: -16.4100\tMean: -15.0000\tDuration: 12.87\t#t_s: 1533.0\n",
      "Epi: 400\t Frame: 522902\tAverage Score: -14.5100\tMean: -20.0000\tDuration: 9.28\t#t_s: 1110.0\n",
      "Epi: 500\t Frame: 705932\tAverage Score: -11.2700\tMean: 3.0000\tDuration: 21.06\t#t_s: 2510.0\n",
      "Epi: 600\t Frame: 940220\tAverage Score: -4.1600\tMean: -7.0000\tDuration: 17.90\t#t_s: 2143.0\n",
      "Epi: 700\t Frame: 1188962\tAverage Score: 4.8800\tMean: -1.0000\tDuration: 22.94\t#t_s: 2755.0\n",
      "Epi: 800\t Frame: 1418125\tAverage Score: 10.1400\tMean: 9.0000\tDuration: 19.41\t#t_s: 2319.0\n",
      "Epi: 900\t Frame: 1626455\tAverage Score: 14.2400\tMean: 17.0000\tDuration: 15.88\t#t_s: 1892.0\n",
      "Epi: 1000\t Frame: 1817176\tAverage Score: 16.2200\tMean: 20.0000\tDuration: 13.73\t#t_s: 1649.0\n",
      "Epi: 1100\t Frame: 2013092\tAverage Score: 16.0300\tMean: 13.0000\tDuration: 18.20\t#t_s: 2175.0\n",
      "Epi: 1200\t Frame: 2198461\tAverage Score: 17.4300\tMean: 19.0000\tDuration: 14.19\t#t_s: 1696.0\n",
      "Epi: 1300\t Frame: 2381727\tAverage Score: 17.7100\tMean: 16.0000\tDuration: 16.75\t#t_s: 2011.0\n"
     ]
    }
   ],
   "source": [
    "train(config, logger)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 119,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[<matplotlib.lines.Line2D at 0x7f6c18098d30>]"
      ]
     },
     "execution_count": 25,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAXwAAAD8CAYAAAB0IB+mAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDIuMi4yLCBodHRwOi8vbWF0cGxvdGxpYi5vcmcvhp/UCwAAIABJREFUeJztnXl8FEX6/z+VyZ0AgQQChGC4b+SICoLKfYgr3qvueqy66qrr7qI/F1fdXa+v6Lq63/2uB673fR+IJ6J4gUCC3DcEQrhCwpGQkGNm6vfHdM9093RPn3Oln/frlVe6q6urajqZTz39VNVTjHMOgiAIou2TEu8GEARBELGBBJ8gCMIlkOATBEG4BBJ8giAIl0CCTxAE4RJI8AmCIFwCCT5BEIRLIMEnCIJwCST4BEEQLiE13g2QUlBQwEtKSuLdDIIgiKSivLy8hnPeWS9fQgl+SUkJysrK4t0MgiCIpIIxtttIPnLpEARBuAQSfIIgCJdAgk8QBOESSPAJgiBcAgk+QRCES7At+IyxYsbYN4yxjYyxDYyxPwjpnRhjixhj24TfHe03lyAIgrCKExa+F8BtnPPBAMYAuJkxNhjAXACLOef9ACwWzgmCIIg4YXsePud8P4D9wnE9Y2wTgCIAswFMELK9BGAJgD/brY8giBAVNQ3Yd/QExvUtAABwzvH+qr2YNqQQn607gItG90BKCgMArNx1GB2y0tC/sJ2hsj9avRfdOmShur4JOempmDiwCwCg6kgjtlUfx8QBXbCu6hj8nKOhxYsfttXg1sn9kJnmUS3P6/Pj/VV7MeqkPFTXNeOUXp3wwaq9mD6kK5ZsrcbsEUVh93DO8cHPezFjaFdkpwfkavOBOjQ0e8E54OfAvqMn0OL144JRRUj1pKC6rgkPf74Ff545AF3aZaLV58cHq/YGn8XaqqNgYNhx6DjSPCkY3qMDenTMwsvLdmPfsROYM7U/5n+7E81eH+pOeNG7cw78HKioOY5rx/dGr4IcAEB1fRN+rjyK6UO64lB9M575bge2HDyOOVP7o+5EK0ryc9AzPxtLd9SgS7tMVNQ0oF+XXLxVtgcT+gfWSD317Q6ke1Jwx4yB6Nsl18Rf3hqOLrxijJUAGAlgOYBCoTMAgAMACjXuuR7A9QDQs2dPJ5tDEG2eiY8uAQDsmjcLALB0Ry1ue2cN8E4ozyWnFAMALn56mSxvJNbvPYY/vLlalibed/b/fo+6Ji92zZuFX/znB1mehmYv7p09VLXMV3/ajb9/vDF4fseMAXjk8y244721AIAh3dujbxd5Z7Ry1xHMeXsNlu88jIcvGg4AmPGv71XLr2/24trxvXDJ/GXYVduItVVHsWjOWfjv9zvxyOdbwBhwcWkxzv3Pj7L7MtNS8MUfz8TfFmwAAOw/2oQFa/ap1rHzUANe/+0YAMCVz63A5gP12Hz/DDy1ZAee/7Ei0OaKwzjR6gs+s8v/uzysnKeW7JCd1za04L3fna5ap5M4NmjLGMsF8B6AP3LO66TXeGCndNXd0jnnz3DOSznnpZ07664MJggiAvVNXtn5wbomS+U0NHs1r9UJdfj84V/p6vpm7TJbfLLzw8dbZOcnWvxh99Q3tQIADh3XLlfkuNCuXbWNAICdNQ0AAm8AANCoqF+kqdUve25ifjWk18Ty/ZzjQF0oXRR7M+wW2hxtHBF8xlgaAmL/Guf8fSH5IGOsm3C9G4BqJ+oiCEIbxuTnVsQH0LDOFDS2hHcKPMKNWQpXj7KtXKVWsU9JYWGXwshMk8uZX2hMc6tf9boUaQcX6bNnpYecIlwo3+vnaPGq38UjPRAJGamxmTDpxCwdBuA5AJs4549JLi0AcJVwfBWAj+zWRRBEZJS6aFXw/QaEqqHZXNnZ6XLBb/bKLXq1KsV2MGXvoIJSNMXymoR6MlLVxxYAoEHSebV4w980RLIknYbYGfl8HK0+9XuUn1ELox2DXZzw4Y8DcAWAdYwx0en3FwDzALzNGLsWwG4AlzhQF0EQCFjXI+9bFJauFMamVh8WbzqI4xFcNGqo6c9Dn27Cxv0hb22DioX/+YYD+Gj1XsweUYTnfqhAdV0TvttWg/+7bCQe/XKrLO/Ly+Txvv701mq8f9PpqG/y4u2yPWjx+bH3SMBVsu1gPYb89XP8ZdYgzTarDRYv2VKNjwV//NIdNeiQlaZ67zUvhoI2rtt7TLOOVZVH8atnf8J9s4cGXVoj7w//O4hEco1J2XesCZxzQx2bHZyYpfMDwg0Lkcl2yycIIpznvq8wZD22+jiufcl8BFo1C3/+dztl56KrRMkf3lyN2SOKcP/C0ADtlMe+1a1zZ00D1lYdw8Ofb8aGfbJhwKBf/q4P1mve7xH8PheN7oF3y6twSWkPXP3CyuD1t8uq8HZZlW479Phxey0m/1P/8wDGLXwAqDzciJPyc6w2yxC00pYgkpAWDReC0vKyai+qjMeq5HHeDcER2aVihI7ZacLvdAdaZA+vz/gzatLoQJ2EBJ8gkhC1GTJA+EBoikUXgREx9xrpFUwScGtYvTfwW+wLtZ5RLPH6jYt4s9faeIsZSPAJIgnxaQiyUiyti6e+WEZDUDkHmMX3ErEDEjurBNB7U52iGfePVUjwCSLONDR7UTL3E8z413ey9MraRkz65xJU1zfhupdW4p2yPcFrfhUhea+8Cje+skqW9ubKPWH5RF5Ztgu3vL4KPj/HhU8txTdbqvG7V8vxxopK2SCmFhc+tVTz2i/+7wfNa5HgsG7h/+WDdfjvdzvx4tJdAEJz+OPJtMe/088kYHSA1w4JtcUhQbiRVZVHAACbD9TL0p//sQI7DzVg4Zr9+GpTNb7aVI2LSwOrZtVc+Le9s8ZUvfd8FFhZ+uB5w1C++whufeNn1Dd58dn6AxY+hZxIM10iwbl1NxQAPPjppuBxufBckwWy8AnCxYj+31RPuAA6OWDKBBVIBJ+3n1t3QymJNO8+GozvW4AXrj7F8v1ac/mdhASfIBIUUYBTU8K/pk6Kc2iwM/6Czzm3ZeFLidXqVZHMNA/SPNbrJMEniDjjtfkl5JxbLkOc0peqEldAa9DWCuJ4QCxcCnpwOGfhR3kNUxgpDEhTeRszSqtGeAYnIR8+QWhwrLEVox5YhJsn9sWcqf2D6SsqDuOS+cvw5Z/O1A01/Ic3V+OnnbVYcdcUAMCJFh8G/fVzAEBedhouHt0D//2+Ipi/9IFFqDnegt+e0QvvlAcWCYnRJKUoB21L5n5i+HOVzP0E6++dHjx/9Msthu+NNje8Uu5YWT9XHnWsLCNwqLvfjNJMFj5BxI/ahmb4/BzPfS9fYfrJWmGp/vYa3TIWrNmH6vrmoLukRhL18Whjq0zsA9cDESSV6Ursul8OHAtF0XxteaWtspKBSMHX5s4ciFsn98NdZ4fCNkwWYv9HYubQrrJzzo3F/NGiNQZvWGThE4QGWpIqppv5crf6/PCkeBwbbHXSpeMGrhhzEgZ1a4+5768Lu3bjWX0ABN7oxFk+men6A75zZw5Ednoq3lslhmuwN/5APnyCSEBErTXz3RZDITg1MKo2D98c7uowPCkpwZ2/tGASNWw2EGWUgYX9D3gSXPDJwicIDZw0osXXdakrxQrHTrQizcNgIkSLKnVN0V/kk0ikepiu9S29biSsdFg8f5tTSu3GEDICWfgEoYnGphYWrOMWnx8/bq/B5c+Gb3dnhpPv/RJnPrLEtoV/wZPaq2TbIi1ev27ABukLwOieHXXLVBP3gtwMAMAAg/sGA6GNYVrs9uIGIAufIDTQsvCDLh0TZbV6OTbss7b6VEnN8eaoRKp0gm4dMrHf5ltMNDijXwHqdEItSC38P0zpj/NH9UB1XRN++cxPqvmVYzgcQNcOmVhy+wT06JiFvnd9JrveqyAHFcK2iFJ++stkLN50ECOK8wx+GuuQhU8QGmgNzgal1sT7e4vPbzkomBqJsEhKDb1pqtGiXUZk27VjTropl44nhaFXQQ56dMqOkF89vaQgB6melLCFX2LYZinZ6R50yErDBaN6oHfn3IjtcwISfILQQM+INmXh+/yOLgRKVAvfTDhgJ9F7tpmpHgOCH54WaRBW2YErI4wq/0SJ8BcjwSdcz+fr92NFxeGwdNFXf7zZi58tBOKSunCcnoGxaX+9fqY4EIuBRzU8OjNwstKNCH749UjF6nUyyk5ZrY+Odb9Ngk+4mmU7anHjq6twyfxlaFTs0Sr9Mp7/5NKgBWf0Szrr36EQwQEL3zkTf+/RE46V5QTFnbIAAL8ec1Jc6tcT8y7tMjC0qH3EPGpFRPqbMSZffKX8t1CulYjXs5FCgk+4GunKV+VmFWGv5MHzwIEZ/fZz69sNJiLtMlNlMX6uHFOCXfNmYfaIIt17n7litK26N903IyxNKswVD52NXfNmBf8+r113GnIyUnFSfg4qHjpbs1yzHTIDw+RBhcEImdr/LwEuGt0Du+bNkrXByowvO5DgE65G6grQs9zFy6FZOsYFwu4c7UREbyGTFnruFz3U4tVIixSFW7T6mco1o0TKHrwm/DYq3U6+6ZmFBJ9wNZG0R2l92Rko9XPepix8wPqqUruCr1avmkuHRbjmBMEOxUYZ5MMniBiiFIMPfq7CJ2v3AwgfGC3fLR+4/Wz9fsP12A2slXDwyJ1lJNTi+5tB7c1CrRMRH7edpx7pXuU1I/sAxxsSfMLVKAX/T2+twc2vB/aFvV2xZeClwgIc8Xv9/Tb9aJkinFvfq9UpxvTu5Gh5Wi6dXwrbMGrfZ7yOqYMLZefiwGu/Lrm4e1YouqXqgKsgycp2PnTBMPQuyFGtb0BhO9w/e0jwvEOWfO78jCGhQVrxf2dkcUe0y0zF7yf1k+V95KLh6F2Qg1sn9cX4vgWq9c27cJhqerQgwSdcjRVj08pAG0f8B20fvnC47TK+v2Ni8FjLVfLwRcNx66S+mmWYsfB/Lyln6uBCLPz9GQCARXPOwnVn9MaTvxql3RYNC/+yU3vi69snqNb3xZ/OxBVjS0Jt9aRg17xZwfOnpQPOQsEdstOw7u/TcWoveYd6SWkxvr59AuZMG4BXrztNtb7zR/ZQTY8WJPiEq5EKhf0IlNr4E2DU1glftrSISL54TwRRN7MLoLTNarWJb1tqTQmOqUbpucf7jc0KJPiEq5EKSkOLsQiSVly1PAGmZTohUNLnFcmHH2nnp0idgRKjbVZzLwV9+FF68NEaDI4mJPiEq5F+aY836wv+/G93yM7rm1rxp7dW41hjK574ZjuWbKlWve/K51dgd2144KxY4oRAiWVwRLbw1fbhFTEzu0daR21Di267pAR9+NGy8KNSanShaJmEq5Eamw0GBP+hzzbjotEhv+tLS3fhg5/3oluHTDy5JNAZSH2+UvS2LYw2etMh83PSI4rqhAGdg8+Lc45Xrj0N0x7/znBd543ojnaZaaamZUrFWjlLSp4vUhnq6c9dVYrKw42G2xJernXJ//dlI9HUoh9z32lI8AlXI108VW9wUxCpS4dJLN5EJ5I+XXZqTyzfWRtR8KcOLpSJXP/CdrhufC88+0N4R5am4qi/uLQY4/oWYNP+OsNt1usbxAF09UVWwm8NW3zyoELVdKPYeXE49+Tutuq2Crl0CFcjnTttxKWjRPzSJ2r0SimRLFLGjC2IMmrVRirLjGWsN+Aa2m4ykM+jMsibhK72qEGCT7ga6cQcIy4dQD4tk5ldVx9HIgo+9AWfgQUtbr2PmxZh0NbMgi2jnYNYpmxWj4rV7yQ0aEsQSYbUMv/ze+uM3SRROzUBHH3/IrxbXmW/cQ4TSWjTPCkRZ9YAgY7O6BTHdpnhm32IG4KoldEpJ121HKMDvKouHcU1p0lCvSfBJ9yNXVdM0KUjeVWobWjBPR+ut1WuGT6+ZbyhfFq+7JN7dMAVY08yFAzOqHU+Y0hXXH9m7+D5/bOHYPRJgX1iM9NCsvPVnDMxojgPT1w+Sr3NkvrU6hafujgtU7Zrlcc5C//TW8/A81eXyttmv9iYQ4JPuBo1udcTCOk90mmKemVEmqpoh4Hd7G0r+OeZA9HHwPZ6DMywhZ+SwnDNuF4AApb9FWNLgvfmpIfmivTt0g4f3jwOJxd30CxHJFLXLGaTuqXEFb1OWPiDu7fHpIHyQV5y6RBEkqEW8MqKMBt5U7AaTlgPw6U6UL2ZIrT0MDvDE5amJZ7SR6a+Y5R8lo40v/h3jJYsJ6Hek+AT7kZtC1Y9y026UUrQwjfgGbIaTliPeFiaRj6v2Cxl1ozUcMHXwvSgrdTCD7p0ouXDTz7FJ8EnXI2aZa43W+XjNfuCx0FRM6CAduPAa2FUdyLNnAGguwUgEBJRLReMrF06tnXX9pmhvBpZ9T5aoVBG3y4Bt5a0gwha+Mmny1GDFl4RrkZNps1Y4lo+fDWcEJ67Zw3CA59sUpQbueD+hbl45KKTkZ0e+ev+t18MQXVdMxZvVg8PAQSs8w9vHoc+ndXDC8vbJfxWufbZH85A53YZwXNNS17nmY3pnY83fjsGvQpy8MaKSrlLR1j85YtiULxkwxELnzH2PGOsmjG2XpLWiTG2iDG2Tfjd0Ym6CMJJ1Cxzj44lLCXWC6+GFulb1kpyMlIxojhPN19mmgen9NKOmS9+1hHFearTLpVE6gwHdWuPgtyQ4Gs9cSMunbF98oNtU7PwvT4SfBGnXDovAlDuLDwXwGLOeT8Ai4Vzgkgo1Iw/MxY+M+HDT3ZL02yfZm6AVz230TLEZysTfKHj9qoN1LgURwSfc/4dgMOK5NkAXhKOXwJwnhN1EYRdth2sD86bV7PMzcymEXNuVMSHUSvBbZZmitaorWpenTJ08Adn64TSxGmZrS577pGI5qBtIedc3PTzAADVSEWMsesZY2WMsbJDhw5FsTkEAWzYdwxTH/8OTwlhjtWMbjOWrChIP1ce1c3bmoSWptTtMkhlvn9pScAFpOpqMmHia1r4BssQXUxnD+sWTBO3Ryxsn6F6jxuJyaAt55wzxlS/RpzzZwA8AwClpaXUFRNRZe+REwCAnysDoXbVZ9cY/zc0MxBrx83/wHlDcd7IIqzfeyxivvK7pwAARj/wlaV6pB9n7d+n4XiTF6fP+xoAMLJn+DDcjKFdUXb3FFnHECzL5CD16r9OxYj7FinaY6yQDllpWHXPVNketDdN6INfnlKs2ja3Ek0L/yBjrBsACL+1h/4JIk6oL+Yxfn+UZlqG0aVdBnIz9O2z/NwM5BjIZ4T2mWnISgvMmRd/q6ElqKFBW2MPNC87PJ4OM6FQnXLSZVNfGWMk9gqiKfgLAFwlHF8F4KMo1kUQJgkIg+1YOhoWqNOLcsyUZ6dq5b1mRVtWlvVmOFoGEcKpaZlvAFgGYABjrIoxdi2AeQCmMsa2AZginBNE3FhXdQzbqo8LZ+KgbXg+DuAbja0KlSTioh4nV96KO1xZmWCktwGJsTIS8AEnMY68+3HOL9O4NNmJ8gnCCX7xnx/C0tR8+CdafPjNCysNlaklSEZW3uoxqFv7sN2h+nUxEuRM2RbjdU4Y0AX/8+nm4LmZmTZKrLwd/Oq0nninrAotPr9QRujaLRP7mm8EIYNCKxCuRk0MT7Qa32s0mj78t24YE5aWn5uB9353OoDAwiK1/XPtWPj9C+Uzcey4dKzw4PnDsPXBmcHnKn07uH36gJi0oS1Dgk+4Gqfi4UeDHI1QCOLApFbblW1ywqdv5TGZCSwXXi/FwYkGJPiEq4nW4lcnipXOOJG6iPS2GXTS7y22wcrnsdOMoIVPgu8oJPiEK1m8uRqLNx2M2iydxhbjbiGz2LGczSJ+OifGJEzVG8PP6CZI8AlXwjlw7Utlti3xeFigRnz0EwZ0Vk3/6zmDdcM03zKxL34/KTBA6klhOKNfAf57ZWnEe9RITWEY37cA868Ybfre+VeMxvi+BcF9cBOJf1w0HOee3D3ezbAEhUcmXI1dyzUeFqiRTubF35yK8t1HcOFTS2Xp14zvhYy0FNz1gfaeu9LBUcYYXrn2NIvtZHj1Omv3ThzQBRMHdLF0b7S5uLQYF5cWx7sZlki87pMgYojfphM/HhEw7c6zV3ZSdubJE8kFCT7hauzqdazi4EsHYu26kZRvNTQw6h5I8AlXcMvrq1TT71u40Va5cXHp2LyfxkHdCwk+4QoWrt2vn8kCRiz8303o42iddqdd0swX90KCTxA2MKKdNzsQEkDqhiEXDGEVEnyCsIERC99pfbbt0iET37WQ4BOEDYwM+jptkduepeNQO4jkgwSfaFPc+/EGDPv7FzGrz4i1zMB0FzuZwaje52QENi3pnpclSycD373QwiuiTfHCj7vC0qLpwjAyj58xIM3DLM3ZL8jNQM3xZlmaUQt/YNf2eOLyUThLseo22fT+9d+ehh552fFuRpuALHyizRNNi9aohqelWPuqjSjOs3SfyKzh3cK2Rkw2H/7pfQrQM58E3wlI8Ik2TzQXRxkpmTEg1RN7lw5BKCHBJ5IWr8+PuqZW+P0cxxpbVfO0eP1oaI5e5EqjPvx0m0HApLU4HVqBcA/kwyeSltvfWYMPV+/DnKn98diirVhxV/iOmpfMX4bVe45GrQ2GpmUyIM1jTvCz0jwRy3OSgtwMZwskEhYSfCJp+XD1PgDAp+sCq2hr6luC1zjnYIxFVewBg9MyYU7wv/jjmShsHxBhNXG3G+xM3K7wN+NK8MtTitHXwD65RNuAXDpE0tMqbHgt9ZN7YxTF0piFb25aZve8TORlp2tetzvDU3w0aZ4UDOza3l5hRFJBgk8kPaK4S33bsQpbbMQfzgB4TPhhdGPl2I6W6UgxRBJCgk8kBDsOHceh+mbVa2v2HEVTq/bAq9cnCr4kLWaCb8yHn2LCLNfLaX+lLSm+WyHBJxKCyf/8FmMeWhyWXl3fhNlP/Ij/9+5azXu9/oBLR6rxPl+sXDr6eRhjuHBUkeEyrRj4BbkZEQd6pYQsfFJ8t0GDtkTCoOaGEadUrqvSHnwVLXypP13sBKKN0Tn+147vhQc+2WQor54Qq7l8frpzkqGy5eWYvoVIckjwiaRHHLT1Sqz6WPnwjVZjJoa9WlZpv6LmHUo1Oe0TII+OGyGXDpH0iP56qbXdmkA+fDuoibLtaZm08sq1kOATCcmRhha8U7YneB5JohpbAm6fxxdtDabFzofvfD26Pnyb39qgD59MfNdBgk8kJLe8sQr/79212FXbYPiexZurg8dmffhn9CswlV8kGsayrg/fZvlik2nQ1n2Q4BMJycG6wBTNVq+1wVezOvzKtadZqseK5+j8kZFn7Oha+A7F0iEL332Q4BNtklj5qa3Uo3eP/jx801XK6xe6Q9J790GCTyQ0Vq3ZWI1LWvHh692h95ntD9oGK7JVDpF8kOATSYFZXfVz4L3yqug0RlGPWex2RnZ1OuTDJ9wGzcMn2iR+znHbO2tiUo9Z9O6Qumz+du4QpKWmYIJkm0K7gn/NuBJsO1iPa8b1slcQkXSQ4BNJgVLk9Pzg0dzlSt4OK/cEbspO9wSnlIpcPLqHzKVTlJeFJy4fJctj16WTl52Op3492lYZRHJCLh0iIRFFkQXPzd7vbHu0sGPhq0XQNGK92x20JdwLCT6RlOjpbCILvqj4HpV9bo1Y73anZRLuhQSfiDlNrT6Mm/c1vt16CH/5YB1K5n4SlkeU0eteLgMQsHx31zZg5H1fYs/hRl0/+G+F+6KNpUFbofVqFn5Ohr6XleSesAoJPhFzKg83Yu/RE3hg4Ua8vrzS0D2cA2+X7cGRxlZ8+PNeXR/+gbomw+15WvBnP3dVKYYWmdsByuszvzBMbLoyRv6UQYW4fXp/3fvJwCesEnXBZ4zNYIxtYYxtZ4zNjXZ9RNtF6u5w0mMzY2hXAMDkQYUYUZwnu6a3wXeLhZXAouArLfzfTeiN7HQDFj4pPmGRqAo+Y8wD4AkAMwEMBnAZY2xwNOsk2j4c0fPRh5cbuaIWKxa+6NJRWPgUxJKINtG28E8FsJ1zvpNz3gLgTQCzo1wnkeBYjeUizc8dsvHTFAOnylJbdaJu2rLwFYIfo4jOhIuJtuAXAdgjOa8S0ggXE4rlYt014ZQ1nKbYOERZrp6PvtmK4Au/U8MsfFJ8IrrEfdCWMXY9Y6yMMVZ26NCheDeHSFCkFr2TuqgUfCV6m6E3e/1hlroeWoO2JPdEtIm24O8FUCw57yGkBeGcP8M5L+Wcl3bu3BkEAUBV/aIxVKl06Sgr1tsqsdXnR0aq2a+R+rRMMvCJaBNtwV8JoB9jrBdjLB3ApQAWRLlOIsExImzKLPIZOtwxcUxNkX8FlPum6Fn4LV7zgq/lw3dqXIIgtIiq4HPOvQBuAfAFgE0A3uacb4hmnUTyYGbQloMHN0WprG10TBxTwwZtzZW7YV8d0s0KvvA7zBVEek9Emaj78Dnnn3LO+3PO+3DOH4x2fUTb5S1hj9v3f96L1ZVHHSkzRdHr9C9sJzvv2Sk7eDxxgLrLsWuHLEN1iW8CY3vnAwDOGd5Ndr1bnrFyRMb07mQqP0FQtEwiKdlZY3yv20goLfqrTy/BA59sAgB8eusZWLazFvcv3IhxffPx8EXDceqDiwEEOoLKw40AgOevKsXRE62ob/KiodmLXz27XLWu8numwufjaJ+VirOHd0NRXhZmDe8GzgPWfncTgr/yrilol0lfX8Ic9B9DxBwr/nelb90p74eyLamSWTuDu7fHyl2HAQB9Ouciw+MJXpMKfn5uBvJ1VuQCQK4kTk6RIO49OmZrZY9I53b69RGEkrhPyyQIIyhXtMZjz1om+baYn5lDEPGH/muJuLH5QL3hvMoVrU7pvZlypP5+imdDJCPk0iESEqUF3xonC18q7NJJNZ6UwEpZMfAaQSQDJPhEzDEy9TE8po35EAZOkyITf4bt/3N2HFtDEOYhlw6RFCiDmDk3aGusJM7l6waU0zkJIhkgwSeSEqciS+oVoyXyyjg4BJEMkEuHiDmv/rRb81pjixfPfl8Br05Y4hUVtU43S5cUDX8+QSQLJPhETDnR4sMbK/ZoXr/hlXJ8v61Gt5wvNhw0VW/7zFTUNXnD0o2O/XJwmciTS4dIRsilQ8QUn47CHm8OF2Un8KQwrL93eli63gA7udKfAAAYlklEQVSyVNaZbFqmUy0jiNhBgk/EFL1B0mhZzhzqbhg9C1/rsjK0MUEkAyT4REzRHSSNYt12OhPl7lzk0iGSERJ8Iqa8skx7wBaIrqtErWy9Ha9ElK4fmqVDJCMk+ERM+ccXWyJej5pLh6vvofvSNadGvE+rNaT3RDJCgk8kFNG08NVEum+XXItlkeITyQcJPpFQOLWgSgnn3JZIKwd3zW5cThCJAAk+kVBEKygah8W3B42byMAnkhESfCJmVB1p1M2zcteRqNVvJaRxe2FXqU456bL0fMU5QSQDtNKWiBk/GFhBmwgsuGVcMDrnL4Z3R0OzDxeOLgIAPHdVKVZUHMYNZ/XRvP/dG8ciI9UDTwpDfVNrTNpMEEYgwSdiRmwi2NuvfHiPvOBxSgrD5af1DJ5PHlSIyYMKI95fWkKbixOJCbl0iJgRoz1LCILQgASfIAjCJZDgEzHDyE5X0aubIAgSfIIgCJdAgk/EjHj68GO16TlBJDIk+ETMIMkliPhCgu8CWrx+bNxXF+9mxBXqbAiCBN8V3L9wI87+9/fYc1h/pWtUSTC3ygUji+LdBIKIKbTwygWs3nMUAHCksQXFnbLj1o5EkvsPbjpdtsCKINwAWfhEzIjvoK38vH1WGkW8JFwHCb6LSDCPSlwhqSfcCAm+CxCDREbS+683H8Teoyei2g6aGkkQ8YUE3wUYsWavebEM5/z7+6i3JV4oV/laCZVMEMkOCT4R5EhjdEP5xtO+V75c0NsG4UZI8F1EvEUukTQ2gZpCEDGDBN8NCO4LLZGLVUcQVwtfeU6KT7gQEnwXIHqrtUTu5WW7bdfxv19tw/bq45rXV+46jFd/sl+PU8T7bYcg4gEJPoG/Ldhg6/5jja14/KutuPSZnzTzXPz0MlTUNNiqBwBG9VRfLHXHjAGmyiG5J9yILcFnjF3MGNvAGPMzxkoV1+5kjG1njG1hjE2310wikRFnwIj7wEaT928ah1NVthC8cmwJCnL1NxYfUNgOAOAnC59wIXZDK6wHcAGA+dJExthgAJcCGAKgO4CvGGP9Oec+m/URFgjNQIyOyMVaO9U2UklhgNcfoSHCJfFZ+KPfNxFEwmHLwuecb+Kcb1G5NBvAm5zzZs55BYDtAE61UxdhHT0fvl3EYmM1tV3tczAw+CMJvpgvOIBNFj7hPqLlwy8CsEdyXiWkETGmqdWHVZVHTd3z5JLtuPP9dZrX/X6OC59aiiueW46bXitXzbNxXx2mPvYt6ppacf/Cjabq1yMnI/zFlDHAgN5HvfMjiERGV/AZY18xxtar/Mx2ogGMsesZY2WMsbJDhw45USQhYcuBetP3PPL5FryxolLzerPXj/LdR/D9thp8uu5AcMaL1MB/bNFWbKs+jp921OK5HypMtyESj158smq6T1D8Z68sxZyp/XHrpL7Ba6JFnyL8x5PgE25E14fPOZ9iody9AIol5z2ENLXynwHwDACUlpbS19BhUiR+FqcernLAM+TSCdXlEYTVZ8TsNknndhk4e1hXfLruQDCNsVBdPfOzMWVwIQDg319vD7RR9OEL3RIN2hJuJFounQUALmWMZTDGegHoB2BFlOoiIiD1qzulcT6l4AfFNIQYeliZN1owsKCIp3u0/61TDASSI4i2it1pmeczxqoAjAXwCWPsCwDgnG8A8DaAjQA+B3AzzdCJD3YGUrUWJ3l9Sgufh9XlEXwnLd7oTIdhipBwjIU6l7TU8H9rLs0IsvAJd2J3ls4HnPMenPMMznkh53y65NqDnPM+nPMBnPPP7DeVsEKKDcU/0Rrqox/6bBNK5n4CAPAq5zQqtLOp1YeP1+wDAMx5e43l+iNR2D5Tds4QetNI82h/5t4FOQCAXJWBX4Jo69B/fRtH7tIxZ9W2eP3IFtYyzf92ZzBd6ZcPnQYqO3YiOlE3P7x5XPD4lkl90b8wF3OF2UTS8YNILp0Hzx+Kc4Z3Q39hARZBuAkKrdDGsTNo26Kyctbn52EuHdE9EtxoJUrekuFFHYLHnXLScempPYPn0t0K01QEX+zsstNTMXlQYXQaSBAJDgm+izDrt271hedv9flVLHz5tMxoLWqK5J2SWfgqPnyCIEjwkwLOOQ7WNZm6x+vzo7q+SWZtcx5wtzQ0e7G9uh57DjeGrU5taPYGj483eXGkoUV2/XBDC3YfblS0T9leU001jNFdqlJpc3KCUIV8+EnAqz/txj0fbcCnt56Bwd3bG7rn/oUb8dKy3Xj7hrHBNM6Bk+/9Upbv9mn9Zeej7l8UPL746aWoa/Li5WtCUTFOn/d1WF3BaZmCztqdez9xQGd8s8X8IrxxffPx4/Za1Y6B5uQQBFn4ScGynbUAgJ012vHmlSzaeBAAcLQxZKGruXSUwtosmUZZ1xSw9tfsiRyaIeTSCQhtxCBmOgzq1h5P/Xq0pXufvfIU/Dh3kiztJaGzolmYBEEWflKgnHNuhBTBrSH1w6sJvhFx1suhHLT12QhFWZSXicw0j6V7s9I9KErPkqV1aZdhuS0E0dYgC7+NIq50lc6ZV7NyjUSY1BvsFYsQuyU7Fj4sdG4RSyN3PkEEIcFPAHYeOo7dtfq7QW3eHwiEtu1gPaqONEbM61Gx8NWE2Ii/XT+LPIPYzkTAzsIzgmhrkOAnAJP++S3O+scS7QyCZv3nm0AgsKmPf4fxD38TsUwPEwU/ZOE/98POsHxq1rtHMctFb8FW0MIX6vzjW6sj5lcSzVk1JPcEEYIEv40SdOlIBF9tT1k1Cz9L4UPXd+kYd+GIIQ3yc0LbEaaqhELYNW+W4TIjQQY+QYQgwW+jpDC1QdvwfGrRLLPSlYIfuS5xmMCIuAZj50syS1fGOi3QRufuE4QbIMFvo4R8+CEL36sSKkFt0FYZi0ZvYNdK5EmpDquFQnAKknuCCEHTMuPI5+sPyPzl1fVN+HLDQfx6zEnBtMraRnyydr/q/ev3HsPGfXUY1qMDPlq9DyOK87CrtgEXjCrCur3HAMjDEzerhCreVRs++Kt0scz/Ltz3L2XO2wGffdWRE/jNC8a2PZC67aPqwycLnyCCkODHkRtfle8He9Orq1C2+wjO7NcZPfOzAQAXPLVU8/5z/u8H1fR5n20OHksDoGWnp6KxRX9bArMrZbceDC0IM7pC9rZpA3DHu2sBaFv4V449CS8v222qLUo8JPgEEYRcOgnEYWFVrFSka4432ypTZuG3GtuDptnrx1n9O9uqVwuxKxnbOx9FeYFFUtL49VJ5vm/2UEwZ1MVWfZ4IsfEJwm2Q4CcQ4kCr2bj1kZC6ceolgdEi0eL1R9WvDsh9+KkR6rLrkqFAagQRggQ/gRC1ycl9v9Vi2uvR7PUhynovE/JIomzXJaNcU0AQboZ8+BbYtL8Ob6yoxL3nDjFsge453Ii/LdiAod3bo1NOOnaqzIkXfeFLd9TgtndWY+bQbmF5xG0GjfL68kpT+QGgqdWPA3X2XElaiC8vKSz0JpMRIX59is2Ohyx8gghBgm+BK55bgZrjzbhlYl90UeytqsXXm6uDP3rc+/FGAMD6vXW22mkHtQiZnXLScVgRH1+Lfl1ycaSxVXMMQhoQ7u/nDsE1L67EkcbWsHn4eqER3rx+DMp3H9G8HsldRBBug74NFlBbPGT0nmTmP5eNBAB0yErTzXvl6SUou3tKWLq4G5b00XVul4GHLhimWo6eS2ZM73zcPLGv5nWy8AkiBAm+BcSFRm7TElF8jSy00hNa6dVIHafd4GfkwyeIECT4FuCK324huCDLwAfXFVqDOmxb8GkePkEEIcG3gBhqQM3SfWDhRlz+35+C5/uOnkCvOz/Bhn3x88c7hUcYQXXGwpdfFzc96ZidLkvvlKPvPopECln4BBGEBm0tELTwVXTv2R8qZOeLN1eDc+Cd8qroN8wmM4d2xWfrD4Slnz2sK26bNgAnhFW6HMAHN52O85/UXgWsZeFL97+VPr6z+nfGvecOwUWje8jy3zZtABpafJgQpYVgBOEmyMK3gChaRkIQJIt9OWFAZ1x3Rm/Va2cP64Y+nXNlPvzenXMjlpeqM59S+VwYY7jq9BLkZMhtkMw0D/7n/GGYNqRr5A9AEIQuJPgWEF0aVqJEJiqcy0McSBH96OJ1P9f30avFuJdCQc0IIvaQ4FtAFPpWHw9Ot+Scy0IRAwFfv9lAZPGCQ1vExVTRh8851x0MNTVLx2AbCYKwBwm+BUTDfuKjS/D4oq0AgLs+XI9+d30my3f1iyvxtwUbYt08S/TvkqvphunSPgNAaCes0pM66Vr4eoOljAHDijoACA3YRpO8bHuDvwTRFiDBt4DUk/P6ikDoArUQBt9tNRYqOBZ8ePO4iNf/PHNgmIgv+tOZ+NcvR2D0SZ0AAF07ZOKFq0/Bvy4doSv4GcIK1x/nTsKpJYH750ztHxyoZWD416Uj8MFNp6NTTrpGKc7w+R/PwFdzzopqHQSRDJDgW4DL5pckhkOiV0FOxOsjivMiXk/zpIS5YfoVtsN5I4tkaRMHdkFh+0zdRWdpQnycorwslBQEYvsXCm8KAAAWiM8/smfHyAU5wMCu7VGQm6GfkSDaOCT4FpC65UVXdrzHIJ0I3WBmkZPeoKtaeGWu8twIgogdbVrwtxyoNyyEWw/Wo6HZi8raRmyvrpcNtm49WI+N++pwvNmLPYcbZdcO1Tfj2InWsDn5FSrRMKOJE2PDVkIpayHf1CRc3UnvCSL2tNmFV6sqj+CCJ5fi7lmDNOeXi6zecxTnPfGjLO3Gs/pg7syB2F5dj2mPfxfx/qmPfRuWNvHRJabbbJXMtBTMHNYV879V33u2cztj7gzpwOapvTrZapN0I/SxffLxVtkeDOjaLrjaiqZlEkTsabOCv+dwYHPuNVXHdPNW1BwPS1u56zAA4FC9fjjg6nrrseN/+PNEjH/4GwBA+d1TMPqBrzTzbrxvOu54dy0WSjY133DvdDAGZKR6cMOZfTDq/kWye9bfOz04hXLjfdMBAJWHG9EuMw0extAxJy24K1ZBbgbK756C1JQUZKZbe/lrl5GK+mavLCzxeSOLML5fgcyPTnJPELGnzQq+6I82sjhKuu+riHhfemp0pam9EGo4K80TFkdGSXZ6KroL+8CKSFemqs12yZVcz04PHA/s2l6WJyM1NC0y3+bgZofsNNQ3e8Pm6SsHTcnAJ4jY02Z9+EHBN+DcbvGF5xHvi7brIU2Y++7j3FCgr0SPqy+Ob+htHq7m1ycIIrq0GQufc45Xl1ciPycdZw/rFpw2+Nn6A/h8/QF4Uhg45xjWowPeK69Cfm4GTu+TjwWr9+GNFeFz6NdUHcMry3ahoqYxqu0WQxAY6ZgA9YBtiYRX+By6K21J7wki5rQZwf9xey3u+XA9AGDt36fJLPMbXy23VOY9H0V3lezAru2CwjhnWv9g+ojiPKxWbDEo6qdU78f2zg8r84azemP+tzsxZVAhdh4KH5uIBj07ZaNSGDO5dVJf3PPRBs1dsbjrdhEgiMTBluAzxv4B4BcAWgDsAPAbzvlR4dqdAK4F4ANwK+f8C5ttjYh071SfjyfsblR3zhyIhz7bjGvH98I95wwGAOyaNyt4XXqshji2cPesQbh2fC+V8gfhzpmDHGyxPt/dMTG4ufoVY0twxdgS3XvIwieI2GPXh78IwFDO+XAAWwHcCQCMscEALgUwBMAMAE8yxqIaMEU68Nrq8yfstL/gQi2L93PJtMZE/YxGIB8+QcQeW4LPOf+Sc+4VTn8CIO5eMRvAm5zzZs55BYDtAE61U5ce0kVDLT5/wg5uikJnt3UklwRBmMVJH/41AN4SjosQ6ABEqoS0qFC++wjuFvz3APDw51uwcZ/+/Pt4IBrlVvsjsSNLVuM+QfthgnAFuoLPGPsKgNp2Q3dxzj8S8twFwAvgNbMNYIxdD+B6AOjZs6fZ2wEEwhtI+XjNPkvl2GXCgM5YsiVyhMzzRhbhw9V7cc34Ekt1XHdGb5TtPoJzT+5u6X4n+fWYnuirs/OVkv9eWYrnf6zQ3GyFIIjooSv4nPMpka4zxq4GcA6AyTzkR9kLoFiSrYeQplb+MwCeAYDS0lJL9l9uRuwnGz1zxWhc/0po9s+s4d3wxOWjACA4gDm2dz6W7ayV3VeQm4GFvz/Dcr3FnbLxya3W73eSB84bZvqeiQO7YOLALlFoDUEQetjy4TPGZgC4A8C5nHPphPUFAC5ljGUwxnoB6AdghZ26IpGdEf0NNJSI4X8jQVMQCYJIJOyaxv8BkAFgkTBj5CfO+Y2c8w2MsbcBbETA1XMz59xnsy5NctJjb+Gnq4T/VUL+aoIgEglbSsk57xvh2oMAHrRTvlFy4mDhZ6bJBT9LZZu+WGzdRxAEYZQ2sdK2e4csXH16CRZtPAiv348jDa1o8fkxYUBn9O2ci/omLzLTUrC84jBSPQzr99ZhxpCu+H7bIZzaK7Q/69qqY6iub8YvS4vh8TC8vrwSt0/rj5E9O+KFHyvw1aZqnNyjAzJSPRhZ3BH/vmwkcjM8WFFxBDecGQrB/PzVpWhq9aO0pCNeXrobJxfnYcGafbhwVNQmKsWdJy4fFZeOlyAI47BEmq9eWlrKy8rK4t0MgiCIpIIxVs45L9XL12ajZRIEQRBySPAJgiBcAgk+QRCESyDBJwiCcAkk+ARBEC6BBJ8gCMIlkOATBEG4BBJ8giAIl5BQC68YY4cA7LZ4ewGAGgebE0uo7fGB2h4fqO3OcxLnvLNepoQSfDswxsqMrDRLRKjt8YHaHh+o7fGDXDoEQRAugQSfIAjCJbQlwX8m3g2wAbU9PlDb4wO1PU60GR8+QRAEEZm2ZOETBEEQEWgTgs8Ym8EY28IY284Ymxvv9ihhjBUzxr5hjG1kjG1gjP1BSO/EGFvEGNsm/O4opDPG2L+Fz7OWMTYqzu33MMZ+ZowtFM57McaWC+17izGWLqRnCOfbheslcW53HmPsXcbYZsbYJsbY2CR65n8S/lfWM8beYIxlJupzZ4w9zxirZoytl6SZfs6MsauE/NsYY1fFse3/EP5n1jLGPmCM5Umu3Sm0fQtjbLokPaE1KAjnPKl/AHgA7ADQG0A6gDUABse7XYo2dgMwSjhuB2ArgMEAHgEwV0ifC+Bh4fhsAJ8BYADGAFge5/bPAfA6gIXC+dsALhWOnwbwO+H4JgBPC8eXAngrzu1+CcB1wnE6gLxkeOYAigBUAMiSPO+rE/W5AzgTwCgA6yVppp4zgE4Adgq/OwrHHePU9mkAUoXjhyVtHyzoSwaAXoLueJJBg4KfLd4NcOAPNhbAF5LzOwHcGe926bT5IwBTAWwB0E1I6wZgi3A8H8BlkvzBfHFoaw8AiwFMArBQ+KLWSL4QwecP4AsAY4XjVCEfi1O7OwiiyRTpyfDMiwDsEcQvVXju0xP5uQMoUYimqecM4DIA8yXpsnyxbLvi2vkAXhOOZdoiPvdk0qC24NIRvxwiVUJaQiK8bo8EsBxAIed8v3DpAIBC4TiRPtO/ANwBwC+c5wM4yjn3CufStgXbLVw/JuSPB70AHALwguCOepYxloMkeOac870AHgVQCWA/As+xHMnx3EXMPueEef4KrkHgjQRIvraH0RYEP2lgjOUCeA/AHznnddJrPGAaJNSUKcbYOQCqOefl8W6LBVIReFV/inM+EkADAq6FIIn4zAFA8HfPRqDT6g4gB8CMuDbKBon6nPVgjN0FwAvgtXi3xSnaguDvBVAsOe8hpCUUjLE0BMT+Nc75+0LyQcZYN+F6NwDVQnqifKZxAM5ljO0C8CYCbp3/BZDHGEtVaVuw3cL1DgBqY9lgCVUAqjjny4XzdxHoABL9mQPAFAAVnPNDnPNWAO8j8LdIhucuYvY5J9LzB2PsagDnAPiV0GEBSdL2SLQFwV8JoJ8wgyEdgUGrBXFukwzGGAPwHIBNnPPHJJcWABBnI1yFgG9fTL9SmNEwBsAxyetxzOCc38k578E5L0HguX7NOf8VgG8AXKTRbvHzXCTkj4tlxzk/AGAPY2yAkDQZwEYk+DMXqAQwhjGWLfzviG1P+Ocuwexz/gLANMZYR+ENZ5qQFnMYYzMQcGOeyzlvlFxaAOBSYVZULwD9AKxAEmhQkHgPIjjxg8DI/1YERsrvind7VNo3HoFX2rUAVgs/ZyPgZ10MYBuArwB0EvIzAE8In2cdgNIE+AwTEJql0xuBf/TtAN4BkCGkZwrn24XrvePc5hEAyoTn/iECsz+S4pkDuBfAZgDrAbyCwMyQhHzuAN5AYKyhFYE3q2utPGcE/OXbhZ/fxLHt2xHwyYvf1acl+e8S2r4FwExJekJrkPhDK20JgiBcQltw6RAEQRAGIMEnCIJwCST4BEEQLoEEnyAIwiWQ4BMEQbgEEnyCIAiXQIJPEAThEkjwCYIgXML/B9yOm8m4qxTsAAAAAElFTkSuQmCC\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "plt.plot(logger.score)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "def watch(config, logger):\n",
    "    env = config.env\n",
    "    env = gym.wrappers.Monitor(env, \"./vid\", video_callable=lambda episode_id: True,force=True)\n",
    "    frame = 0 \n",
    "    \n",
    "    print(\"loading file from: {}\".format(logger.log_file_path))\n",
    "\n",
    "    agent = Agent(channel_in=4, action_size=env.action_space.n, config=config)\n",
    "    agent.qnetwork_local.load_state_dict(torch.load(logger.log_file_path, map_location='cpu'))\n",
    "\n",
    "    for i_episode in range(1):\n",
    "        states = env.reset()\n",
    "        scores = 0\n",
    "\n",
    "        input()\n",
    "\n",
    "        for t in count():\n",
    "            actions = agent.act(states, network_only=True)\n",
    "            next_states, rewards, dones, _ = env.step(np.array(actions))\n",
    "            env.render(\"human\")\n",
    "            states = next_states\n",
    "            scores += rewards\n",
    "            number_of_time_steps = t\n",
    "            frame += 1\n",
    "\n",
    "            if np.any(dones):\n",
    "                break \n",
    "        \n",
    "        print(\"Score: {}\".format(scores))\n",
    "    env.close()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 120,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "loading file from: checkpoint-score17_56.pth\n",
      "\n",
      "Score: 20.0\n"
     ]
    }
   ],
   "source": [
    "logger.log_file_path = \"checkpoint-score17_56.pth\"\n",
    "print(logger.log_file_path)\n",
    "\n",
    "watch(config, logger)"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.8"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}
